# -*- coding: utf-8 -*-
"""
Top360buy: 这个程序用于京东商城(www.360buy.com)的贝叶斯排序
Copyright 2010 liuzhongshu (http://www.liuzhongshu.com)

用法:
 top360buy.py [options]
 命令行参数请使用 --help 查看
 举例：
  python top360buy.py -c 平板电视
"""

from optparse import OptionParser
from urllib2 import urlopen
from urlparse import urljoin
from BeautifulSoup import BeautifulSoup
import unittest, time, re, sys
import httplib, codecs
import StringIO,csv
import sys,os,re,time,subprocess
import traceback
import zlib
import socket
import Queue
import threading


DEFAULT_TIME_OUT = 60
DEFAULT_AVERAGE_MULTIPLY = 5.0
SYSTEM_ENCODING = sys.stdin.encoding if sys.stdin.encoding else "utf-8"
CSV_EXPORT_ENCODING = "gbk"
BUY_360_ENCODING = "gbk"
BUY_360_WEBSITE = "http://www.360buy.com"
BUY_360_PRICESITE = "http://price.360buy.com"
PAGE_URL_ALL_CATALOG = "/allSort.aspx"

P_BAYES_RATE,P_ID,P_URL,P_CATALOG,P_NAME,P_PRICE,P_PROMOTION,P_RATE,P_COUNT,P_HIS_COUNT,P_RECENT_COUNT = range(11)
T_ID, T_URL, T_CATALOG, T_NAME, T_PRICE, T_PROMOTION, T_RATE, T_COUNT = range(8)

parser = OptionParser()
parser.add_option('-c', '--catalog', dest='catalog',default=u"all",
                  help=u'设置商品类别，缺省为: all')
parser.add_option('-n', '--number', dest='number',type="int", default=10,
                  help=u'设置显示Top多少商品，缺省为: 10')     
parser.add_option('-i', '--import', dest='importcsv',default="",
                  help=u'设置导入历史csv文件，用于生成销量变化数据。')
parser.add_option('-e', '--export', dest='exportcsv',default="",
                  help=u'设置导出csv文件，缺省为空，表示不导出')
parser.add_option('-r', '--recent', action="store_true", dest='use_recent',
                  help=u'设置使用最近销量代替总销量参与排序，需要和-i配合使用。')
parser.add_option('-d', '--dryrun', action="store_true", dest='dryrun',
                  help=u'设置只显示类别，不抓取数据')
parser.add_option("-f", '--factor', dest='factor',type="int",default=0,
                  help=u'手动设置平均评价数，缺省0，表示自动计算')
parser.add_option("-w", '--workers', dest='workers',type="int",default=1,
                  help=u'设置线程数，缺省1个')
parser.add_option("-v", '--verbose', action="store_true", dest="verbose",
                  help=u'设置调试模式，显示更多信息')                                      
parser.add_option("-p", '--promotion', action="store_true", dest="promotion",
                  help=u'设置只抓取有促销的商品')                                      
parser.add_option("-s", '--silent', action="store_true", dest="silent",
                  help=u'设置安静模式')                                      
parser.add_option("-t", '--timeout', dest='timeout',type="int",default=DEFAULT_TIME_OUT,
                  help=u'设置网络超时，单位为秒，缺省为60')
                
(options, args) = parser.parse_args(sys.argv[1:])

class util:
    @staticmethod
    def start_thread(func, *args):
        thread = threading.Thread(target=func, args=args)
        thread.setDaemon(True)
        thread.start()
        return thread

    @staticmethod
    def write(*args, **kwd):
        if args:
            fout = kwd.get("file", sys.stdout)
            encoding = kwd.get("encoding", SYSTEM_ENCODING)
            sep = kwd.get("sep", " ")
            
            for idx,a in enumerate(args[0:]):
                if isinstance(a,unicode):
                    fout.write(a.encode(encoding))
                else:
                    fout.write(str(a))
                
                if idx < len(args) -1:
                    fout.write(sep)
                    
            fout.write(kwd.get("end", "\n"))
            #in macos, flush is needed
            fout.flush()

    @staticmethod
    def log(*args, **kwd):
        if args and (not options.silent):
            util.write(*args, **kwd)
                        
    @staticmethod
    def load_page(url):
        page = urlopen(url)
        data = page.read()
        try:
            decompress_data = zlib.decompress(data, 16+zlib.MAX_WBITS)
        except zlib.error:
            return data
        
        return decompress_data
        
        
class Top360buy():
    
    def add_one_product(self, id, url, catalog, name, price, promotion, rate, buy_count):
        if options.verbose:
            util.log( name, price, buy_count, rate)

        if self.history_products.has_key(id):
            history_buy_count = self.history_products[id]
        else:
            history_buy_count = 0
        recent_buy_count = buy_count - history_buy_count
            
        self.products.append([0, id, url, catalog, name, price, promotion, rate, buy_count, history_buy_count, recent_buy_count])
        self.total_buy_count += buy_count
        self.total_rate += buy_count * rate
        self.total_products +=1
    
    def update_promotion(self, url, products):
        promotion_src = util.load_page(url)
        for product in products:
            r_search = re.search('"Pid":' + product[T_ID] + ',"PF":\\[(.*?)\\]', promotion_src)
            if not r_search is None:
                product[T_PROMOTION] = r_search.group(1)
            
    #遍历一个小类下的所有产品  
    def buy360_test_catalog(self, catalog, url):
        products = []                   
        try:   
            catalog_home_src = util.load_page(url)
            catalog_rate_url=urljoin(url, re.search(u'href="([^"]*)"[^>]*>评论数</a>', catalog_home_src.decode(BUY_360_ENCODING)).group(1))
                
            if options.verbose:
                util.log( "getting products:", catalog_rate_url)
            catalog_rate_src =  util.load_page(catalog_rate_url)
        except Exception:
            util.log( "error in catalog home:", catalog)
            traceback.print_exc()
            return
            
        try:
            totalpage = int(re.search('<div class="pagin pagin-m fr"><span class="text">[0-9]+/([0-9]+)', catalog_rate_src).group(1))
        except Exception:
            totalpage = 1

        if options.verbose:
            util.log( "total page:", totalpage)
                
        #遍历所有产品
        for page in range(totalpage):
            util.log( '.', end='')
            catalog_rate_url = re.sub("-[0-9]+\.html","-" + str(page+1) + ".html", catalog_rate_url)
            if options.verbose:
                util.log( "getting products:", catalog_rate_url)
            
            try:    
                catalog_rate_src = util.load_page(catalog_rate_url)
            except Exception:
                util.log( "getting products page error")
                continue
            
            page_products=[]            
            promotion_url = BUY_360_PRICESITE + "/PromotionFlag.aspx?pid="
            for product in re.findall("<div\\s+class='p-name'>.*?<span\\s+id='", catalog_rate_src, re.DOTALL):
                try:
                    url = re.search("href='(.*?)'", product).group(1)
                    id = re.search("([0-9]+)\\.html", url).group(1)
                    promotion_url = promotion_url + str(id) + ","
                    name_tag = re.search("<div\\s+class='p-name'>.*?</div>", product).group(0)
                    price_tag = re.search("<div[^>]*?class='p-price'>.*?</div>", product).group(0)
                    evaluate_tag = re.search("<span\\s+class='evaluate'>.*?</span>", product).group(0)
                    reputation_tag = re.search("<span\\s+class='reputation'>.*?</span>", product).group(0)
                    name = re.sub('<.*?>','',name_tag).decode(BUY_360_ENCODING)
                    price = re.sub('<.*?>','',price_tag).decode(BUY_360_ENCODING)
                    buy_count = int(re.search("[0-9]+", re.sub('<.*?>','',evaluate_tag)).group(0))
                    good_rate = int(re.search("([0-9]+)%", re.sub('<.*?>','',reputation_tag)).group(1)) / 100.0
                    page_products.append([id, url, catalog, name, price, '', good_rate, buy_count])
                    if options.verbose:
                        util.log("find product:" + name)
                except Exception, e:
                    util.log( "something in html regex broken")
                    #traceback.print_exc()                                                
            
            self.update_promotion(promotion_url, page_products)
            for product in page_products:
                if (not options.promotion) or len(product[T_PROMOTION]) > 0: 
                    products.append(product)
                        
        #遍历完成
        util.log(' ')
        return products
        
        
    def worker(self, catalogQ, productQ):
        while True:
            try:
                catalogList = catalogQ.get()
            except Queue.Empty:
                break
            
            try:
                util.log( "getting ", catalogList[0], catalogList[1])
                results = self.buy360_test_catalog(catalogList[0], catalogList[1])
            except Exception:
                results = []
                
            productQ.put(results)
            catalogQ.task_done()
    
                
    def main(self):
        self.catalogs = {}
        self.products=[]
        self.history_products={}
        self.total_products=0.0
        self.total_buy_count=0.0
        self.total_rate=0.0
        
        if options.importcsv != "":
            csvreader = csv.reader(open(options.importcsv, "rb"))
            for row in csvreader:
                self.history_products[row[P_ID]] = int(row[P_COUNT])
                                     
        socket.setdefaulttimeout(options.timeout)
        user_catalog = options.catalog.decode(SYSTEM_ENCODING)               
        #构造所有需要测试的类别
        util.log( "connect to: ", BUY_360_WEBSITE)
        pagesrc = util.load_page(urljoin(BUY_360_WEBSITE,PAGE_URL_ALL_CATALOG))
        soup = BeautifulSoup(pagesrc)
        soup_allsort = soup.find("div",id="allsort")
        soup_large_cat = [ s for s in soup_allsort.findAll("h2") if len(s.findAll(text=options.catalog.decode(SYSTEM_ENCODING))) != 0]
        soup_small_cat = [ s for s in soup_allsort.findAll("dt") if len(s.findAll(text=options.catalog.decode(SYSTEM_ENCODING))) != 0]
        if options.catalog=="all":
            for catalog in soup_allsort.findAll("em"):
                self.catalogs[catalog.a.string] = urljoin(BUY_360_WEBSITE, catalog.a["href"])
        elif len(soup_large_cat) != 0:
            soupCatlog = soup_allsort.find("h2", text=options.catalog.decode(SYSTEM_ENCODING))
            for catalog in soup_large_cat[0].parent.parent.findAll("em"):
                    self.catalogs[catalog.a.string] = urljoin(BUY_360_WEBSITE, catalog.a["href"])            
        elif len(soup_small_cat) != 0:
            for catalog in soup_small_cat[0].parent.findAll("em"):
                    self.catalogs[catalog.a.string] = urljoin(BUY_360_WEBSITE, catalog.a["href"])            
        elif len(soup_allsort.findAll(text=re.compile(options.catalog.decode(SYSTEM_ENCODING)))) > 0:
            for catalog in soup_allsort.findAll("em"):
                if catalog.a.string.find(options.catalog.decode(SYSTEM_ENCODING)) != -1:
                    self.catalogs[catalog.a.string] = urljoin(BUY_360_WEBSITE, catalog.a["href"])
        
        if len(self.catalogs) == 0:                    
            util.log( u"必须指定一个类别")
            return

        util.log( u'类别:' + ' '.join(self.catalogs.keys()))
        if options.dryrun:
            return
               
        catalogQ = Queue.Queue()
        productQ = Queue.Queue()
        for idx,catalog in enumerate(self.catalogs):
            catalogQ.put([catalog, self.catalogs[catalog]])

        catalog_count = len(self.catalogs)
        for i in range(min(catalog_count, options.workers)):
            util.start_thread(self.worker, catalogQ, productQ)
        
        #等待所有线程完成
        while catalog_count:
            products = productQ.get()
            catalog_count -= 1
            if not products is None:
                for product in products:
                    self.add_one_product(product[0], product[1], product[2], product[3], product[4], product[5], product[6], product[7])            
                            
        if  self.total_products < 1:
            util.log( u"找不到任何商品")
            return
        
        if options.factor == 0:       
            average_buy_count = self.total_buy_count / self.total_products * DEFAULT_AVERAGE_MULTIPLY
        else:
            average_buy_count = options.factor
            
        average_rate = self.total_rate / self.total_buy_count       
        util.log( u"总共:",self.total_products,self.total_buy_count)
        util.log( u"平均:",average_buy_count,average_rate)
           
        for product in self.products:
            if options.use_recent:            
                product[P_BAYES_RATE] = ( (average_buy_count * average_rate) + (product[P_RATE] * product[P_RECENT_COUNT] )) / ( average_buy_count + product[P_RECENT_COUNT])
            else:
                product[P_BAYES_RATE] = ( (average_buy_count * average_rate) + (product[P_RATE] * product[P_COUNT] )) / ( average_buy_count + product[P_COUNT])

        util.log( "*"*80 )
        if options.exportcsv != "":
            writer = csv.writer(open(options.exportcsv, "wb"))
            for product in self.products:
                writer.writerow([product[P_BAYES_RATE],
                                 product[P_ID],
                                 product[P_URL],   
                                 product[P_CATALOG].encode(CSV_EXPORT_ENCODING),
                                 product[P_NAME].encode(CSV_EXPORT_ENCODING),
                                 product[P_PRICE].encode(CSV_EXPORT_ENCODING),
                                 product[P_PROMOTION],
                                 product[P_RATE],
                                 product[P_COUNT],
                                 product[P_HIS_COUNT],
                                 product[P_RECENT_COUNT]])
                
        for idx,product in enumerate(sorted(self.products, reverse=True)):
            if idx >= options.number:
                break
                
            if options.use_recent:                    
                util.write( idx + 1, product[P_NAME], product[P_PRICE], '{' + product[P_PROMOTION] + '}', product[P_RATE], '/',product[P_RECENT_COUNT], ':', product[P_BAYES_RATE], product[P_URL])
            else:
                util.write( idx + 1, product[P_NAME], product[P_PRICE], '{' + product[P_PROMOTION] + '}', product[P_RATE], '/',product[P_COUNT], ':', product[P_BAYES_RATE], product[P_URL])    
              

if __name__ == "__main__":
    Top360buy().main()





